/*
Copyright (c) 2011, Stephane Albert
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the organization nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL STEPHANE ALBERT BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#include "debridFetcherBase.h"

std::map<std::string, std::string> debridFetcherBase::htmlChars;

debridFetcherBase::debridFetcherBase() {
    logLevel = CALL_LOG;
    maxRetries = 5;
    mustExit = false;
    mustStopDL = false;
    fileList = NULL;
    curlHandle = NULL;
    fetchThread = NULL;
    initCURL();
}

debridFetcherBase::~debridFetcherBase() {
    //Better be shure everything is dead before we free cURL
    stop();
    //We're safe now
    unloadCURL();
    if (fileList != NULL)
        delete fileList;
}

bool debridFetcherBase::initCURL() {
    unloadCURL();
    curlHandle = curl_easy_init();
    if (curlHandle != NULL)
        return true;
    return false;
}

bool debridFetcherBase::loadCURLSettings() {
    if (curlHandle != NULL) {
        curl_easy_reset(curlHandle);
        curl_easy_setopt(curlHandle, CURLOPT_ENCODING, "");         //Set default encoding to accept compressed pages (mandatory for filecrop)
        curl_easy_setopt(curlHandle, CURLOPT_COOKIEFILE, "");       //Set auto cookies
        curl_easy_setopt(curlHandle, CURLOPT_MAXREDIRS, MAX_REDIRS);       //Follow location trap (deadlock)
        return true;
    }
    return false;
}

void debridFetcherBase::unloadCURL() {
    if (curlHandle != NULL) {
        curl_easy_cleanup(curlHandle);
        curlHandle = NULL;
    }
}

void debridFetcherBase::setMaxRetries(int newMaxRetries) {
    if (newMaxRetries < MAX_RETRIES)
        maxRetries = newMaxRetries;
    else
        maxRetries = MAX_RETRIES;
}

int debridFetcherBase::getMaxRetries() const {
    return maxRetries;
}

//FIXME need a better error handling
void debridFetcherBase::setLogFile(const std::string& newLogFile) {
    if (logFile.is_open())
        logFile.close();
    logFile.open(newLogFile.c_str(), std::ios_base::out|std::ios_base::app);
    if (!logFile.good())
        logFile.close();
}

void debridFetcherBase::setFailedLogFile(const std::string& newFailedLogFile) {
    if (logFailedFile.is_open())
        logFailedFile.close();
    logFailedFile.open(newFailedLogFile.c_str(), std::ios_base::out|std::ios_base::app);
    if (!logFailedFile.good())
        logFailedFile.close();
}

void debridFetcherBase::setLogLevel(int newLogLevel) {
    if (logLevel > FALL_LOG)
        logLevel = FALL_LOG;
    else if (logLevel < 0)
        logLevel = NO_LOG;
    else
        logLevel = newLogLevel;
}

void debridFetcherBase::logMessage(const std::string& msg, int flags) {
    if (logLevel) {
        int logMSB;
        std::ostream *curStream = NULL;
        //Get logLevel MSB to skip unused log options
        for (logMSB=FMSG_LOG; !(logMSB & logLevel); logMSB >>= 1);
        for (int curFlag=logMSB; curFlag != LFFAIL_LOG; curFlag >>= 1) {
            if (logLevel & curFlag) {
                if (flags & MISC_ERR) {     //Errors Flag
                    switch (curFlag) {
                        case CFAIL_LOG:
                            curStream = &std::cerr;
                            break;
                        case FFAIL_LOG:
                            curStream = &logFile;
                            break;
                        default:
                            continue;
                    }
                }
                else {                      //Standard output
                    switch (curFlag) {
                        case CMSG_LOG:
                            curStream = &std::cout;
                            break;
                        case FMSG_LOG:
                            curStream = &logFile;
                            break;
                        default:
                            continue;
                    }
                }
                if (flags == DECODE) {
                    *curStream << _("Decode: ");
                }
                else if (flags == DEBRID) {
                    *curStream << _("Debrid: ");
                }
                else if (flags == DOWNLOAD) {
                    *curStream << _("Downloading: ");
                }
                else if (flags == DECODE_ERR) {
                    *curStream << _("Error decoding: ");
                }
                else if (flags == DEBRID_ERR) {
                    *curStream << _("Error debrid: ");
                }
                else if (flags == DOWNLOAD_ERR) {
                    *curStream << _("Error downloading: ");
                }
                else if (flags == MISC_ERR) {
                    *curStream << _("Error: ");
                }
                *curStream << msg << std::endl;
                curStream->flush();
            }
        }
    }
}

void debridFetcherBase::logMessage(const std::string& msg, const std::string& linkURL, int flags) {
    if (logLevel)
        if (flags & MISC_ERR)
            if (logLevel & LFFAIL_LOG)
                logFailedFile << linkURL << std::endl;
    std::string newMsg = linkURL + ", " + msg;
    logMessage(newMsg, flags);
}

void debridFetcherBase::setMonitorDir(const std::string& newMonitorDir) {
    if (!newMonitorDir.empty()) {
        monitorDirectory = newMonitorDir;
        if (newMonitorDir[newMonitorDir.size()-1] != DIR_SEP)
            monitorDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getMonitorDir() const {
    return monitorDirectory;
}

void debridFetcherBase::setWorkingDir(const std::string& newWorkingDir) {
    if (!newWorkingDir.empty()) {
        workingDirectory = newWorkingDir;
        if (newWorkingDir[newWorkingDir.size()-1] != DIR_SEP)
            workingDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getWorkingDir() const {
    return workingDirectory;
}

void debridFetcherBase::setDoneDir(const std::string& newDoneDir) {
    if (!newDoneDir.empty()) {
        doneDirectory = newDoneDir;
        if (newDoneDir[newDoneDir.size()-1] != DIR_SEP)
            doneDirectory.push_back(DIR_SEP);
    }
}

std::string debridFetcherBase::getDoneDir() const {
    return doneDirectory;
}

size_t debridFetcherBase::dataToStr(void *rawBuffer, size_t size, size_t nmemb, void *strPtr) {
    if (rawBuffer == NULL)
        return 0;
    std::string *outputString = (std::string*)strPtr;
    size_t initLen = outputString->length();
    outputString->append((char*)rawBuffer, nmemb);
    return outputString->length() - initLen;
}

size_t debridFetcherBase::dataToFile(void *rawBuffer, size_t size, size_t nmemb, void *filePtr) {
    if (rawBuffer == NULL)
        return 0;
    std::ofstream *outputFile = (std::ofstream*)filePtr;
    if (!outputFile->good())
        return 0;
    std::streampos initPos = outputFile->tellp();
    outputFile->write((const char*)rawBuffer, nmemb);
    return outputFile->tellp() - initPos;
}

size_t debridFetcherBase::dataToFileFFS(void *rawBuffer, size_t size, size_t nmemb, void *ffs_ptr) {
    fetchFileStruct *ffs = (fetchFileStruct *)ffs_ptr;
    if (ffs->fetcher->needToStop())
        return 0;
    return dataToFile(rawBuffer, size, nmemb, ffs->file);
}

int debridFetcherBase::seekFile(void *instream, curl_off_t offset, int origin) {
    if (instream == NULL)
        return CURL_SEEKFUNC_FAIL;
    std::ofstream *outputFile = (std::ofstream*)instream;
    if (!outputFile->good())
        return CURL_SEEKFUNC_FAIL;
    switch (origin) {
        case SEEK_SET: outputFile->seekp(offset, std::ios_base::beg);
            break;
        case SEEK_CUR: outputFile->seekp(offset);
            break;
        case SEEK_END: outputFile->seekp(offset, std::ios_base::end);
            break;
        default:
            return CURL_SEEKFUNC_FAIL;
    }
    return CURL_SEEKFUNC_OK;
}

std::string debridFetcherBase::postData(const std::string& URL, const std::string& postData) {
    if (curlHandle != NULL) {
        std::string outputData;
        loadCURLSettings();
        curl_easy_setopt(curlHandle, CURLOPT_URL, URL.c_str());
        curl_easy_setopt(curlHandle, CURLOPT_POSTFIELDS, postData.c_str());
        curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, dataToStr);
        curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &outputData);
        CURLcode retCode = curl_easy_perform(curlHandle);
        if (retCode == CURLE_OK)
            return outputData;
    }
    return "";
}

std::string debridFetcherBase::getData(const std::string& URL, bool follow) {
    if (curlHandle != NULL) {
        std::string outputData;
        loadCURLSettings();
        curl_easy_setopt(curlHandle, CURLOPT_URL, URL.c_str());
        curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, dataToStr);
        curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &outputData);
        if (follow)
            curl_easy_setopt(curlHandle, CURLOPT_FOLLOWLOCATION, 1);
        CURLcode retCode = curl_easy_perform(curlHandle);
        if (retCode == CURLE_OK)
            return outputData;
    }
    return "";
}

std::string debridFetcherBase::decodeWuploadFolder(const std::string& folderURL) {  //Same as filesonic
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<caption>Files Folder</caption>");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<a href=\"", linksPos);
    if (curLinkPos == std::string::npos || curLinkPos > linksEndPos)
        return decodedURL;
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList(retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeMegauploadFolder(const std::string& folderID) {
    std::string folderURL = "http://www.megaupload.com/xml/folderfiles.php/?folderid=";
    std::string decodedURL;
    folderURL += folderID;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linkPos = retPage.find("url=\"");
    if (linkPos == std::string::npos)
        return decodedURL;
    linkPos += 5;
    size_t linkEndPos = retPage.find('"', linkPos);
    decodedURL = retPage.substr(linkPos, linkEndPos-linkPos);
    linkPos = retPage.find("url=\"", linkPos);
    while (linkPos != std::string::npos) {
        linkPos += 5;
        linkEndPos = retPage.find('"', linkPos);
        insertStrInFList(retPage.substr(linkPos, linkEndPos-linkPos));
        linkPos = retPage.find("url=\"", linkPos+1);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFileserveFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<table class=\"file_list\">");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<td class=\"m35\">", linksPos);
    if (curLinkPos == std::string::npos)
        return decodedURL;
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = "http://www.fileserve.com";
    decodedURL += retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<td class=\"m35\">", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList("http://www.fileserve.com"+retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<td class=\"m35\">", curLinkPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFilesonicFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t linksPos = retPage.find("<caption>Files Folder</caption>");
    if (linksPos == std::string::npos)
        return decodedURL;
    size_t linksEndPos = retPage.find("</table>", linksPos);
    size_t curLinkPos = retPage.find("<a href=\"", linksPos);
    if (curLinkPos == std::string::npos)
        return decodedURL;
    curLinkPos += 9;
    size_t curLinkEndPos = retPage.find('"', curLinkPos);
    decodedURL = retPage.substr(curLinkPos, curLinkEndPos-curLinkPos);
    curLinkPos = retPage.find("<a href=\"", curLinkPos);
    while (curLinkPos < linksEndPos && curLinkPos != std::string::npos) {
        curLinkPos += 9;
        curLinkEndPos = retPage.find('"', curLinkPos);
        insertStrInFList(retPage.substr(curLinkPos, curLinkEndPos-curLinkPos));
        curLinkPos = retPage.find("<a href=\"", curLinkPos);
    }
    return decodedURL;
}

//FIXME due to insertion links are in reverse, maybe doing it top down next
std::string debridFetcherBase::decode4sharedFolder(const std::string& folderURL) {
    std::string decodedURL;
    std::string retPage = getData(folderURL);
    if (retPage.empty())
        return decodedURL;
    size_t findEndPos;
    size_t findPos = retPage.find("&sId=");
    if (findPos == std::string::npos)
        return decodedURL;
    findPos += 5;
    findEndPos = retPage.find("\" language=\"", findPos);
    if (findEndPos == std::string::npos)
        return decodedURL;
    std::string sId = retPage.substr(findPos, findEndPos-findPos);
    findPos = retPage.find("<td class=\"filelistname\" >");
    std::string foundString;
    bool foundFirst = false;
    while (findPos != std::string::npos) {
        findPos = retPage.find("href=\"", findPos);
        findPos += 6;
        findEndPos = retPage.find('"', findPos);
        foundString = retPage.substr(findPos, findEndPos-findPos);
        if (foundString.find("javascript:openNewWindow('") != std::string::npos) {   //New link
            foundString.erase(0, 26);
            foundString.erase(foundString.length()-2, 2);
            if (!foundFirst) {
                decodedURL = foundString;
                foundFirst = true;
            }
            else
                insertStrInFList(foundString);
        }
        else if (foundString.find("javascript:changeDir(") != std::string::npos) {
            srand(time(NULL));
            foundString.erase(0, 21);
            foundString.erase(foundString.length()-1);
            std::string newFolder = "http://www.4shared.com/account/changedir.jsp?sId=";
            newFolder.append(sId);
            newFolder.append("&ajax=true&changedir=");
            newFolder.append(foundString);
            newFolder.append("&random=0.");
            newFolder.append(intToString(rand()));
            std::string newFolderPage = getData(newFolder);
            if (!newFolderPage.empty()) {
                size_t findFolderPos = newFolderPage.find("AccountFacade.currentDirPass");
                if (findFolderPos != std::string::npos) {
                    findFolderPos += 32;
                    size_t findFolderEndPos = newFolderPage.find("';", findFolderPos);
                    foundString = newFolderPage.substr(findFolderPos, findFolderEndPos-findFolderPos);
                    newFolder = "http://www.4shared.com/dir/";
                    newFolder.append(foundString);
                    if (!foundFirst) {
                        decodedURL = decode4sharedFolder(newFolder);
                        foundFirst = true;
                    }
                    else
                        insertStrInFList(decode4sharedFolder(newFolder));
                }
            }
        }
        findPos = retPage.find("<td class=\"filelistname\" >", findEndPos);
    }
    return decodedURL;
}

std::string debridFetcherBase::decodeFilecrop(const std::string& linkURL) {
    std::string decodedURL;
    std::string linkPage = getData(linkURL);
    //TODO find a way to bypass captcha
    size_t linkPos = linkPage.find("/captcha.php?id");
    if (linkPos != std::string::npos) {    //Captcha requested
        linkPage.insert(linkPos, "http://www.filecrop.com");
        std::string access = doneDirectory + "access.html";
        std::ofstream accessFile;
        accessFile.open(access.c_str());
        if (accessFile.is_open()) {
            accessFile << linkPage;
            std::string errMSg = _("unable to decode link, captcha is required check page in: ");
            errMSg += access;
            errMSg += _(".\nThen restart the download.");
            logMessage(errMSg, linkURL, MISC_ERR);
        }
        return linkURL;
    }
    linkPos = linkPage.find("<a class=\"linkmaincolor\"");
    if (linkPos != std::string::npos) {     //Single link
        linkPos += 58;
        size_t linkEndPos = linkPage.find('"', linkPos);
        if (linkEndPos != std::string::npos) {
            decodedURL = linkPage.substr(linkPos, linkEndPos-linkPos);
        }
    }
    else if ((linkPos = linkPage.find("<table class=clear>")) != std::string::npos) {   //Multiple links
        std::string curLink = "http://www.filecrop.com";
        bool firstPage = true;
        do {
            if (!firstPage) {
                --linkPos;
                size_t begLinkPos = linkPage.rfind('"', linkPos);
                ++begLinkPos;
                std::string newPageURL = "http://www.filecrop.com/"+linkPage.substr(begLinkPos, linkPos-begLinkPos+1);
                linkPage = getData(newPageURL);
                linkPos = linkPage.find("<table class=clear>");
            }
            do {
                linkPos = linkPage.find("\" href=\"", linkPos);
                linkPos += 8;
                curLink.resize(23);
                curLink += linkPage.substr(linkPos, linkPage.find('"', linkPos)-linkPos);
                if (firstPage) {
                    decodedURL = curLink;
                    firstPage = false;
                }
                else {
                    insertStrInFList(curLink);
                }
                linkPos = linkPage.find("<table class=clear>", linkPos);
            } while (linkPos != std::string::npos);
        } while ((linkPos = linkPage.find("\">Next</a></td>")) != std::string::npos);
    }
    else
        return linkURL;
    return decodedURL;
}

std::string debridFetcherBase::decodeLink(const std::string& linkURL) {
    if (fileList == NULL)
        return linkURL;
    std::string decodedURL;
    if (linkURL.find("wupload.") != std::string::npos) {
        if (linkURL.find("/folder/") != std::string::npos) {
            decodedURL = decodeWuploadFolder(linkURL);
        }
        else {
            decodedURL = linkURL;
            size_t remPos = decodedURL.find("/w867/");
            if (remPos != std::string::npos)
                decodedURL.erase(remPos, 5);
            return decodedURL;
        }
    }
    else if (linkURL.find("megaupload.") != std::string::npos) {
        size_t folderPos = linkURL.find("/?f=");
        if (folderPos != std::string::npos) {
            decodedURL = decodeMegauploadFolder(linkURL.substr(folderPos+4));
        }
        return decodedURL;
    }
    else if (linkURL.find(".rapidshare.") != std::string::npos) {
        size_t linkPos = linkURL.find("#!download");    //New version, AD hates it !
        if (linkPos != std::string::npos) {
            decodedURL = "https://rapidshare.com/files/";
            linkPos = linkURL.find('|', linkPos);
            linkPos = linkURL.find('|', linkPos+1);
            ++linkPos;
            size_t linkEndPos = linkURL.find('|', linkPos+1);
            decodedURL.append(linkURL.substr(linkPos, linkEndPos-linkPos));
            decodedURL.append("/");
            linkPos = linkEndPos;
            ++linkPos;
            linkEndPos = linkURL.find('|', linkPos);
            decodedURL.append(linkURL.substr(linkPos, linkEndPos-linkPos));
        }
        return decodedURL;
    }
    else if (linkURL.find("fileserve.") != std::string::npos) {
        if (linkURL.find("/list/") != std::string::npos) {
            decodedURL = decodeFileserveFolder(linkURL);
        }
        return decodedURL;
    }
    else if (linkURL.find("4shared.") != std::string::npos) {
        if (linkURL.find("/dir/") != std::string::npos) {
            decodedURL = decode4sharedFolder(linkURL);
        }
        return decodedURL;
    }
    else if (linkURL.find("filesonic.") != std::string::npos) {
        if (linkURL.find("/folder/") != std::string::npos)
            decodedURL = decodeFilesonicFolder(linkURL);
        size_t remPos = decodedURL.find("/file/r");
        if (remPos != std::string::npos)
            decodedURL.erase(remPos+5, 11);
        return decodedURL;
    }
    else if (linkURL.find("filecrop.") != std::string::npos) {
        decodedURL = decodeFilecrop(linkURL);
    }
    else if (linkURL.find("filestube.") != std::string::npos) {
        std::string linkPage = getData(linkURL);
        size_t linksPos = linkPage.find("id=\"copy_paste_links\"");
        if (linksPos != std::string::npos) {
            linksPos = linkPage.find("http://", linksPos);

            size_t linksEndPos = linkPage.find("</pre> </div><script", linksPos);
            if (linksEndPos == std::string::npos)
                return linkURL;
            size_t curLinkEndPos = linkPage.find("http://", linksPos+1);
            if (curLinkEndPos > linksEndPos) {  //Only one link
                decodedURL = linkPage.substr(linksPos, linksEndPos-linksPos);
            }
            else {                              //Mutliple links
                decodedURL = linkPage.substr(linksPos, curLinkEndPos-linksPos);
                linksPos = curLinkEndPos;
                insertStrInFList(linkPage.substr(linksPos, linksEndPos-linksPos));
            }
            for (int i=decodedURL.length()-1; decodedURL[i] == '\r' || decodedURL[i] == '\n' || decodedURL[i] == ' '; --i) {    //Sanitize URL
                decodedURL.erase(i, decodedURL.length()-i);
            }
        }
        else
            return linkURL;
    }
    else if (linkURL.find("debridor.") != std::string::npos) {
        decodedURL = linkURL;
        if (linkURL[7] == 'w')
            decodedURL.erase(7, 4);
        std::string linkPage = getData(decodedURL);
        size_t linkPos = linkPage.find("<input name=\"get\"");
        if (linkPos != std::string::npos) {
            linkPos += 39;
            size_t linkEndPos = linkPage.find('"', linkPos);
            if (linkEndPos != std::string::npos) {
                std::string postStr = "get=";
                postStr.append(linkPage.substr(linkPos, linkEndPos-linkPos));
                postData("http://debridor.com/dl.php", postStr.c_str());
                char *newURL;
                if (curl_easy_getinfo(curlHandle, CURLINFO_REDIRECT_URL, &newURL) == CURLE_OK)
                    if (newURL == NULL)
                        return decodedURL;
                    decodedURL = newURL;
            }
        }
        else
            return decodedURL;
    }
    else if (linkURL.find("upsafe.org/") != std::string::npos) {
        decodedURL = linkURL;
        decodedURL.replace(linkURL.find("upsafe.org/"), 10, "debridor.com");
    }
    else
        return linkURL;

    decodedURL = decodeLink(decodedURL);
    return decodedURL;
}

std::string debridFetcherBase::urlEncode(const std::string& rawURL) {
    std::string encodedURL;
    if (curlHandle != NULL) {
        char *escapedURL;
        escapedURL = curl_easy_escape(curlHandle, rawURL.c_str(), rawURL.length());
        if (escapedURL != NULL) {
            encodedURL = escapedURL;
            curl_free(escapedURL);
        }
    }
    return encodedURL;
}

std::string debridFetcherBase::urlDecode(const std::string& rawURL) {
    std::string decodedURL;
    if (curlHandle != NULL) {
        char *unescapedURL;
        unescapedURL = curl_easy_unescape(curlHandle, rawURL.c_str(), rawURL.length(), NULL);
        if (unescapedURL != NULL) {
            decodedURL = unescapedURL;
            curl_free(unescapedURL);
        }
    }
    return decodedURL;
}

int debridFetcherBase::getFile(const std::string& fileURL) {
    if (curlHandle == NULL)
        return CONN_ERR;
    //Log current download
    logMessage(fileURL, DOWNLOAD);
    //Extract filename from URL and open it
    size_t begPos = fileURL.rfind('/');
    if (begPos == std::string::npos)
        return URL_ERR;
    ++begPos;
    std::string filePath = workingDirectory;
    filePath.append(htmlDecode(fileURL.substr(begPos)));
    std::ofstream outFile(filePath.c_str(), std::ios_base::binary|std::ios_base::out);
    if (!outFile.is_open())
        return SYS_ERR;
    //Prepare struct to allow download canceling
    fetchFileStruct ffs;
    ffs.fetcher = this;
    ffs.file = &outFile;
    //Init cURL
    loadCURLSettings();
    curl_easy_setopt(curlHandle, CURLOPT_URL, fileURL.c_str());
    curl_easy_setopt(curlHandle, CURLOPT_WRITEFUNCTION, dataToFileFFS);
    curl_easy_setopt(curlHandle, CURLOPT_WRITEDATA, &ffs);
    curl_easy_setopt(curlHandle, CURLOPT_SEEKFUNCTION, seekFile);
    curl_easy_setopt(curlHandle, CURLOPT_SEEKDATA, &outFile);
    if (logLevel & CMSG_LOG)
        curl_easy_setopt(curlHandle, CURLOPT_NOPROGRESS, 0);

    //Stalled transfert detection
    curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_LIMIT, MIN_SPEED);               //Stalled Transfert detection 1Kb/s
    #ifdef DEBUG
        curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, 10);
    #else
        curl_easy_setopt(curlHandle, CURLOPT_LOW_SPEED_TIME, MIN_SPEED_TIMEOUT);    //< 1 Kbps for 60 seconds
    #endif

    int funcRetCode = CONN_ERR;     //If we quit loop it's because something is wrong with the connection
    CURLcode retCode;
    std::streamoff resumePos=0;
    for (int retry=maxRetries; retry > 0; --retry) {
        retCode = curl_easy_perform(curlHandle);
        if (mustStopDL) {
            funcRetCode = CNCL_ERR;
            break;
        }
        /*
         * Stalled transfert check
         */
        else if (retCode == CURLE_OPERATION_TIMEDOUT) {
            resumePos = outFile.tellp();
            if (resumePos > 0) {        //Some bytes written, trying to resume
                logMessage(_("Transfert stalled, trying to resume."), MISC_ERR);
                curl_easy_setopt(curlHandle, CURLOPT_RESUME_FROM, resumePos);
                continue;
            }
        }
        /*
         * An error occurred
         */
        else if (retCode != CURLE_OK) {
            //Size checking (from response header)
            double contentLength;
            if (curl_easy_getinfo(curlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength) == CURLE_OK) {
                if (contentLength > 0) {        //If contentLength == -1 skip
                    if (outFile.tellp()-resumePos != contentLength) {     //We need to download more
                        resumePos = outFile.tellp();
                        logMessage(_("Transfert interrupted, trying to resume."), MISC_ERR);
                        curl_easy_setopt(curlHandle, CURLOPT_RESUME_FROM, resumePos);
                        continue;
                    }
                }
            }
            //If we can't set resuming position then we retry from the beginning
            resumePos = 0;
        }
        /*
         * No error
         */
        else {
            //Size checking
            double contentLength;
            funcRetCode = NO_ERR;
            if (curl_easy_getinfo(curlHandle, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength) == CURLE_OK) {
                if (contentLength > 0) {        //If contentLength == -1 skip
                    if (outFile.tellp()-resumePos != contentLength) {
                        std::string errorMessage = _("Size mismatch, expected: ");
                        errorMessage.append(doubleToString(contentLength+resumePos));
                        errorMessage.append(_(" found: "));
                        errorMessage.append(doubleToString(outFile.tellp()));
                        logMessage(errorMessage, MISC_ERR);
                        funcRetCode = SIZE_ERR;
                    }
                    else {		//If size match
                        //Close it so we flush buffers
                        outFile.close();
                        if (checkFileForError(filePath))
                            funcRetCode = HOST_ERR;
                    }
                }
            }
            break;
        }
        logMessage(_("Error during transfert, retrying."), MISC_ERR);
    }
    return funcRetCode;
}

void debridFetcherBase::dlEachFile() {
    logMessage(_("Downloading list."));
    std::string curFile;
    std::string decodedURL;
    std::string dlLink;
    std::string filePath;
    int retCode;
    size_t begPos;
    bool debridDone = false;
    while (fileList->good() && !mustExit) {
        getline(*fileList, curFile);
        for (int i=curFile.length()-1; curFile[i] == '\r' || curFile[i] == '\n' || curFile[i] == ' '; --i) {    //Sanitize URL
            curFile.erase(i);
        }
        if (curFile.length() < 3) { //Possible new lines, dirty
            continue;
        }
        if (!(debridDone = isDebrid(curFile))) {
            decodedURL = decodeLink(curFile);
            if (decodedURL.empty()) {
                logMessage("", curFile, DECODE_ERR);
                continue;
            }
        }

        for (int retry=maxRetries; retry > 0; --retry) {
            if (!debridDone) {
                dlLink = debridLink(decodedURL);
                if (dlLink.compare(0, 7, "http://")) {     //If the URL doesn't begin with http:// then it's an error
                    logMessage(dlLink, decodedURL, DEBRID_ERR);
                    break;
                }
            }
            else {   //File is already ready -> direct download
                dlLink = curFile;
            }
            retCode = getFile(dlLink);
            if (retCode == HOST_ERR && !debridDone) {
                std::string errorMsg = _("server side error, trying to debrid the link again.");
                logMessage(errorMsg, DOWNLOAD_ERR);
            }
            else
                break;
        }

        filePath = workingDirectory;
        begPos = dlLink.rfind('/');
        ++begPos;
        filePath.append(htmlDecode(dlLink.substr(begPos)));

        if (retCode != NO_ERR) {
            std::string errorMsg;
            switch (retCode) {
                case URL_ERR:
                    errorMsg = _("incorrect URL.");
                    break;
                case CONN_ERR:
                    errorMsg = _("connection problem.");
                    break;
                case CNCL_ERR:
                    if (mustExit)
                        errorMsg = _("transfert cancelled, stopping thread.");
                    else
                        errorMsg = _("transfert cancelled.");
                    break;
                case SYS_ERR:
                    errorMsg = _("filesystem problem, unable to create temporary file.");
                    break;
                case HOST_ERR: {
                    if (debridDone)
                        errorMsg = _("server side error, maybe the link is too old.");
                    else
                        errorMsg = _("server side error, exceeded maximum attempt.");
                }
                    break;
                case SIZE_ERR:
                    errorMsg = _("size error.");
                    break;
            }

            //TODO depending on the error code keep the file as incomplete
            remove(filePath.c_str());

            if (debridDone)
                logMessage(errorMsg, curFile, DOWNLOAD_ERR);
            else
                logMessage(errorMsg, decodedURL, DOWNLOAD_ERR);
        }
        else {
            std::string newFilePath = doneDirectory;
            newFilePath.append(htmlDecode(dlLink.substr(begPos)));
            if (rename(filePath.c_str(), newFilePath.c_str()) != 0) {
                //Rename/Moving failed, file must be in a different FS: copying
                std::ifstream oldFile;
                std::ofstream newFile;
                oldFile.open(filePath.c_str(), std::ifstream::binary);
                if (!oldFile.is_open())
                    continue;
                newFile.open(newFilePath.c_str(), std::ifstream::binary);
                if (!newFile.is_open())
                    continue;
                newFile << oldFile.rdbuf();
                oldFile.close();
                remove(filePath.c_str());
            }
        }
        debridDone = false;
        //Be sure that we canceled what we wanted. And don't suppress the flag because the copy was too long for example.
        if (retCode == CNCL_ERR)
            mustStopDL = false;
    }
    logMessage(_("Finished downloading list."));
}

void debridFetcherBase::processDownloadList(const std::string& downloadList) {
    if (fileList != NULL)   //Shouldn't happen
        delete fileList;
    fileList = getFileContent(downloadList);
    if (fileList != NULL) {
        if (remove(downloadList.c_str()) != 0) {
            logMessage(_("Unable to remove file, stopping thread."), MISC_ERR);
            mustExit = true;
        }
        else
            dlEachFile();
        delete fileList;
        fileList = NULL;
    }
    /*else    //FIXME could produce a deadlock
        logMessage(_("Unable to open a list from watchdir."), MISC_ERR);*/
}

std::stringstream *debridFetcherBase::getFileContent(const std::string& filePath) {
    if (filePath.empty())
        return NULL;
    std::ifstream inputFile(filePath.c_str());
    if (!inputFile.good())
        return NULL;
    std::stringstream *fileContent = new std::stringstream;
    *fileContent << inputFile.rdbuf();
    inputFile.close();
    return fileContent;
}

bool debridFetcherBase::checkDelBadFile(const std::string& fileName) {
    //Skips . and ..
    if (fileName.at(0) == '.') {    //Skipping hidden files and deleting the ones created by MacOS
        if (fileName == ".DS_Store") {            //Stupid MacOS .DS_Store
            std::string filePath = monitorDirectory + fileName;
            if (remove(filePath.c_str()) != 0) {
                //TODO maybe not fatal and just skip.
                logMessage(_("Unable to remove .DS_Store file, stopping thread."), MISC_ERR);
                mustExit = true;
            }
        }
        else if (fileName.length() > 1 && fileName.at(1) == '_') {         //MacOS generate ._ description files which tamper with the program behaviour
            std::string filePath = monitorDirectory + fileName;
            if (remove(filePath.c_str()) != 0) {
                logMessage(_("Unable to remove ._* file, stopping thread."), MISC_ERR);
                mustExit = true;
            }
        }
        return true;
    }
    return false;
}

void *debridFetcherBase::threadEntry(void *data) {
    ((debridFetcherBase*)data)->watchDirectory();
    return 0;
}

bool debridFetcherBase::createThread() {
    if (fetchThread == NULL) {
        fetchThread = new pthread_t;
        if (!pthread_create(fetchThread, NULL, threadEntry, (void*)this))
            return true;
    }
    return false;
}

bool debridFetcherBase::deleteThread() {
    if (fetchThread != NULL) {
        if (!pthread_join(*fetchThread, NULL)) {
            delete fetchThread;
            fetchThread = NULL;
            return true;
        }
    }
    return false;
}

bool debridFetcherBase::needToStop() {
    return mustStopDL;
}

void debridFetcherBase::insertStrInFList(const std::string& str) {
    std::stringstream endList;
    std::streampos curPos;
    curPos = fileList->tellg();
    if (fileList->eof())    //Removes EOF Flag
        fileList->clear();
    endList << fileList->rdbuf();   //If we are at the end it'll pass the eof flag to endList
    fileList->seekp(curPos, std::ios_base::beg);
    *fileList << str << std::endl;  //Finish with a newline to prepare for the next insertion
    if (endList.good())
        *fileList << endList.rdbuf();
    fileList->seekg(curPos, std::ios_base::beg);
}

bool debridFetcherBase::start() {
    mustExit = false;
    mustStopDL = false;
    return createThread();
}

bool debridFetcherBase::cancel() {
    mustStopDL = true;
    return true;
}

void debridFetcherBase::stop(bool forceExit) {
    if (fetchThread != NULL) {
        mustExit = true;
        if (forceExit)
            cancel();
        deleteThread();
    }
}

std::string debridFetcherBase::htmlEncode(const std::string& rawString) {
    if (htmlChars.empty())
        initHtmlChars();
    std::string result = rawString;
    std::map<std::string, std::string>::iterator curPos = htmlChars.begin();
    std::map<std::string, std::string>::iterator endPos = htmlChars.end();
    size_t lastFound;
    while (curPos != endPos) {
        lastFound = result.find(curPos->first);
        while (lastFound != std::string::npos) {
            result.replace(lastFound, curPos->first.size(), curPos->second);
            lastFound += curPos->second.size();
            lastFound = result.find(curPos->first, lastFound);
        }
        ++curPos;
    }
    return result;
}

std::string debridFetcherBase::htmlDecode(const std::string& rawString) {
    if (htmlChars.empty())
        initHtmlChars();
    std::string result = rawString;
    std::map<std::string, std::string>::iterator curPos = htmlChars.begin();
    std::map<std::string, std::string>::iterator endPos = htmlChars.end();
    size_t lastFound;
    while (curPos != endPos) {
        lastFound = result.find(curPos->second);
        while (lastFound != std::string::npos) {
            result.replace(lastFound, curPos->second.size(), curPos->first);
            lastFound += curPos->first.size();
            lastFound = result.find(curPos->second, lastFound);
        }
        ++curPos;
    }
    return result;
}

void debridFetcherBase::initHtmlChars() {
    htmlChars.clear();
    htmlChars.insert(std::pair<std::string, std::string>("&", "&amp;"));
    htmlChars.insert(std::pair<std::string, std::string>("<", "&lt;"));
    htmlChars.insert(std::pair<std::string, std::string>(">", "&gt;"));
    htmlChars.insert(std::pair<std::string, std::string>("\"", "&quot;"));
    htmlChars.insert(std::pair<std::string, std::string>("'", "&#039;"));
    htmlChars.insert(std::pair<std::string, std::string>(" ", "&nbsp;"));
    htmlChars.insert(std::pair<std::string, std::string>("£", "&pound;"));
    htmlChars.insert(std::pair<std::string, std::string>("à", "&agrave;"));
    htmlChars.insert(std::pair<std::string, std::string>("ç", "&ccedil;"));
    htmlChars.insert(std::pair<std::string, std::string>("è", "&egrave;"));
    htmlChars.insert(std::pair<std::string, std::string>("é", "&eacute;"));
    htmlChars.insert(std::pair<std::string, std::string>("ê", "&ecirc;"));
    htmlChars.insert(std::pair<std::string, std::string>("ù", "&ugrave;"));
    htmlChars.insert(std::pair<std::string, std::string>("©", "&copy;"));
    htmlChars.insert(std::pair<std::string, std::string>("Ã", "&Atilde;"));
}
